#!/usr/bin/env python
# coding=utf8


import sys

sys.path.append('..')
sys.path.append('../../')
from i_util.pybeanstalk import PyBeanstalk
import json
import traceback

from thrift import Thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.transport.TTransport import TMemoryBuffer
from thrift.protocol.TBinaryProtocol import TBinaryProtocol
from bdp.i_crawler.i_extractor.ttypes import BaseInfo, CrawlInfo, ExtractInfo, PageParseInfo
import time
from pymongo import MongoClient
import hashlib
from bson.objectid import ObjectId


def main():
    host = '101.201.100.58'
    port = 27017
    database = 'crawl_data'
    coll = 'gov_enterprise_data_liaoning'
    client = MongoClient(host, port)
    db = client[database][coll]
    min_time = "2016-09-12 00:00:00"
    max_time = "2016-09-18 00:00:00"
    min_ms = time.mktime(time.strptime(min_time, '%Y-%m-%d %H:%M:%S'))
    max_ms = time.mktime(time.strptime(max_time, '%Y-%m-%d %H:%M:%S'))
    print min_ms, max_ms
    # param = {'shareholder_information': {"$ne": []}, 'key_persion': {"$ne": []}, 'invested_companies': {"$ne": []}}
    param = {"branch": {"$ne": []}, "key_persion": {"$ne": []}, "shareholder_information": {"$ne": []}}
    cursor = db.find()
    num = 0

    try:
        for item in cursor:
            num += 1
            if num >= 0:
                src_url = item.get("src_url", "")

                data = get_param_data(item)
                data = json.dumps(data)
                site_id = get_md5_i64(item.get("site_id", ""))
                site = src_url.split('//')[-1].split('/')[0]
                base_info = BaseInfo(url=src_url, site_id=site_id, site=site)
                extract_info = ExtractInfo(ex_status=2, extract_data=data, topic_id=49)
                tube = "extract_info_gsxx"
                send_msg(base_info, extract_info, tube)

            if num >= 10:
                break

            if num % 500 == 0:
                print num
        else:
            print "no result"
    except:
        print traceback.format_exc()

    print "finish sumnum:%s" % num


def get_param_data(item):
    '''获取入参'''
    company = item.get("_id", "")
    if isinstance(company, ObjectId):
        company = item.get("company_name", "")
    if not company or isinstance(company, ObjectId):
        company = ''
    province = item.get("province", "")
    src_registered_capital = item.get("registered_capital", "0.0")

    business_scope = item.get("business_scope", "")
    legal_man = item.get("legal_man", "")
    trademark = item.get("trademark", [])

    period = item.get("period", "")
    registered_date = item.get("registered_date", "")
    registered_code = item.get('registered_code', '')
    address = item.get("address", "")
    enterprise_type = item.get("enterprise_type", "")

    key_person1 = item.get("key_person", [])
    key_person2 = item.get("key_persion", [])

    key_person_list = []
    for person in key_person1:
        try:
            if isinstance(person, list):
                if len(person) == 2:
                    person_map = {"key_person_name": person[0], "key_person_position": person[1]}
                    key_person_list.append(person_map)
        except:
            print traceback.format_exc()
    for person in key_person2:
        try:
            if isinstance(person, list):
                if len(person) == 2:
                    person_map = {"key_person_name": person[0], "key_person_position": person[1]}
                    key_person_list.append(person_map)
        except:
            print traceback.format_exc()

    key_person_list = remove_duplicate_dict_2(key_person_list, "key_person_name", "key_person_position")

    registered_address = item.get("registered_address", "")
    business_status = item.get("business_status", "")

    shareholder_information = item.get("shareholder_information", [])
    shareholder_information_list = []
    num = 0
    for each in shareholder_information:
        try:
            if isinstance(each, list):
                num += 1
                if num == 1 or (len(each) > 0 and len(each) < 8):
                    continue
                shareholder_information_map = {"shareholder_name": each[0], "shareholder_type": each[1],
                                               "subscription_amount": each[2], "subscription_type": each[3],
                                               "subscription_time": each[4], "paied_amount": each[5],
                                               "paied_time": each[6],
                                               "paied_type": each[7],
                                               "certificate_type": "", "certificate_no": "", "detail": "", }
                shareholder_information_list.append(shareholder_information_map)
        except:
            print traceback.format_exc()

    shareholder_information_list = remove_duplicate_dict(shareholder_information_list, "shareholder_name")

    changerecords = item.get("changerecords", [])
    changerecords_list = []
    for each in changerecords:
        try:
            if isinstance(each, dict):
                changerecords_map = {"change_date": each.get("change_date", ""),
                                     "change_item": each.get("change_item", ""),
                                     "after_content": each.get("after_content", ""),
                                     "before_content": each.get("before_content", "")}
                changerecords_list.append(changerecords_map)
        except:
            print traceback.format_exc()

    changerecords_list = remove_duplicate_dict(changerecords_list, "change_date")

    hezhun_date = item.get("hezhun_date", "")

    branch = item.get("branch", [])
    branch_list = []
    for each in branch:
        try:
            branch_map = {}
            if isinstance(each, basestring):
                branch_map = {"company_name": each,
                              "registered": "",
                              "type": ""}
            elif isinstance(each, dict):
                branch_map = {"company_name": each.get("compay_name", ""),
                              "registered": each.get("registered_code", ""),
                              "type": each.get("type", "")}

            branch_list.append(branch_map)
        except:
            print traceback.format_exc()

    branch_list = remove_duplicate_dict(branch_list, "company_name")

    invested_companies = item.get("invested_companies", [])
    invested_companies_list = []
    for each in invested_companies:
        try:
            if isinstance(each, dict):
                invested_companies_map = {"change_time": each.get("change_time", ""), "name": each.get("name", ""),
                                          "legal_man": each.get("legal_man", ""),
                                          "register_code": each.get("register_code", ""),
                                          "start_year": each.get("start_year", ""),
                                          "status": each.get("status", ""),
                                          "invest_amount": each.get("invest_amount", "")}
        except:
            print traceback.format_exc()

        invested_companies_list.append(invested_companies_map)
    invested_companies_list = remove_duplicate_dict(invested_companies_list, "name")

    investor_change = item.get("investor_change", [])
    electronic_business_site = item.get("electronic_business_site", "")
    gov_site = item.get("website", "")

    if src_registered_capital:
        src_registered_capital = src_registered_capital.encode('utf8')
    else:
        src_registered_capital = '0.0'

    data = {
        "company": company,
        "province": province,
        "business_scope": business_scope,
        "legal_man": legal_man,
        "trademark": trademark,
        "period": period,
        "registered_date": registered_date,
        "registered_code": registered_code,
        "address": address,
        "enterprise_type": enterprise_type,
        "key_person": key_person_list,
        "registered_address": registered_address,
        "business_status": business_status,
        "shareholder_information": shareholder_information_list,
        "changerecords": changerecords_list,
        "hezhun_date": hezhun_date,
        "branch": branch_list,
        "invested_companies": invested_companies_list,
        "investor_change": investor_change,
        "electronic_business_site": electronic_business_site,
        "gov_site": gov_site,
        "src_registered_capital": src_registered_capital,

    }
    return data


import ctypes


def get_md5_i64(obj):
    m = hashlib.md5()
    m.update(obj)
    return ctypes.c_int64(int(m.hexdigest()[8:-8], 16)).value


def remove_duplicate_dict(data_map_list, key):
    '''字典list去重'''
    data_set = set()
    new_data_map_list = []
    for data_map in data_map_list:

        if data_map[key] not in data_set:
            data_set.add(data_map[key])
            new_data_map_list.append(data_map)
    return new_data_map_list


def remove_duplicate_dict_2(data_map_list, first_key, second_key):
    '''字典list去重'''
    data_set = set()
    new_data_map_list = []
    for data_map in data_map_list:
        if data_map[first_key] == "" and data_map[second_key] == "":
            continue
        if (data_map[first_key], data_map[second_key]) not in data_set:
            data_set.add((data_map[first_key], data_map[second_key]))
            new_data_map_list.append(data_map)
    return new_data_map_list


def send_msg(base_info, extract_info, tube):
    '''往消息队列中发送消息'''
    crawl_info = CrawlInfo()

    pybeanstalk = PyBeanstalk('101.201.102.37')

    req = PageParseInfo(base_info=base_info, crawl_info=crawl_info, extract_info=extract_info, scheduler="a",
                        parse_extends="b", data_extends="c")
    tMemory_b = TMemoryBuffer()
    tBinaryProtocol_b = TBinaryProtocol(tMemory_b)
    req.write(tBinaryProtocol_b)
    body = tMemory_b.getvalue()
    try:
        pybeanstalk.put(tube, body)
        # print "send beanstalk msg success"
    except:
        print traceback.format_exc()


if __name__ == "__main__":
    print "start"
    main()
    # test_map_list = [{"name":"张三","positon":"董事"},{"name":"张三","positon":"董事"},{"name":"李四","positon":"董事"}]
    # new_list = remove_duplicate_dict(test_map_list,"name")
    # for i in new_list:
    #    for key,value in i.items():
    #        print key, value
